from functools import *
from toolz.curried import *
from languagetools import *
from importlib import reload
from timeit import timeit
import operator

# mano
import delfi
import pickler
import article_ns
import languagetools

import numpy as np
import pandas as pd
from scipy import stats, integrate

import matplotlib
matplotlib.use('Qt5Agg')
import matplotlib.pyplot as plt
import seaborn as sns

###

articles = pipe( 
        pickler.pickles_frm_to((2016,11),(2016,12)),
        map(pickler.unpickle),
        list,
        pd.DataFrame )
body_features = articles['body'].apply(
        lambda b: pipe(
            dict(body=b),
            article_ns.features,
            pd.Series ))
main_categories = articles \
        .category_chain.apply(first) \
        .to_frame('main_category')

###

X = pd.concat( [ body_features, main_categories ], axis=1)
#groups = X.groupby('main_category')
outly_factor = lambda X: np.abs(X - X.mean()) / X.std()
remove_outliers = curry( lambda factor,X: X[outly_factor(X) < factor] )
do_distplot = lambda X: sns.distplot(X, hist=False, label=name)
for name, group in X.groupby('main_category'):
    if len(group) > 15:
        pipe( group.sents_in_para,
                remove_outliers(3),
                do_distplot )
        
###

from minisom import MiniSom

input_len = len(body_features.columns)
som = MiniSom(20,20,input_len)
som.train_random(body_features.as_matrix(), 100)
calc_coord = curry( lambda som, serie: \
        pipe( serie.values, som.winner, pd.Series ) )
coords = pipe(
        body_features.apply(calc_coord(som), axis=1),
        lambda df: df.rename( columns=lambda x: 'coord' + str(x) )
        )

out = pd.concat( [ coords, main_categories ], axis=1 )

jitter = lambda X: X + ( np.random.randn(len(X)) * 0.25 )

jittered = pd.DataFrame(
        dict(coord0=jitter(coords.coord0),
            coord1=jitter(coords.coord1),
            main_category=main_categories.main_category) )

###

        (
        sns.jointplot('coord0','coord1',data=jittered, space=0)
        .plot_joint(sns.kdeplot, zorder=0, n_levels=10, shade=True)
        )

###

from ggplot import *

add_emphasis = \
lambda data, cats: \
    pipe( data,
            #lambda X: X.main_category.isin(cats),
            #lambda emph_cats: pd.concat([data.main_category, emph_cats]),
            lambda X: X.main_category.apply(
                lambda x: x if x in cats else '(other)' ),
            lambda X: X.to_frame('emphasis'),
            lambda X: pd.concat([ data, X ], axis=1)
            ) # nejaugi sito negalima pakeisti kazkuom?

#add_emphasis(jittered, ['news','verslas'])

plot_emphasis = lambda data, cats: (                                   
        ggplot( aes(x='coord0',y='coord1',color='emphasis'),
            data=add_emphasis(data, cats) )
        + geom_point()
        )
#plot_emphasis(jittered, ['perpasauli','auto','gyvenimas'])

###

import plotly.plotly as py
import plotly.graph_objs as go

data = pd.concat([ jittered, articles.title ], axis=1)

trace_group = lambda name, data: \
    go.Scatter(
            x = data.coord0,
            y = data.coord1,
            mode = 'markers',
            name = name,
            text = data.title
            )

apply_tuple = curry( lambda f, t: f(*t) )

pipe( data.groupby('main_category'),
        map(apply_tuple(trace_group))
        list,
        py.plot )

#py.plot([trace])
